﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Common.Dao;
using Common.Model.ReportModels.CrawlerData;
using Common.Redis;
using GetReport.Models.CrawlerData;
using ikvm.extensions;
using log4net;
using Quartz.Util;
using WebCrawlerHelper.Controllers;

namespace Spider
{
    public class IpProxyPool
    {
        private static readonly ILog Log = LogManager.GetLogger(typeof(IpProxyPool));

        private static readonly List<ProxyInfo> IpList = new List<ProxyInfo>();

        private static readonly ConcurrentDictionary<string, List<ProxyInfo>> Pool =
            new ConcurrentDictionary<string, List<ProxyInfo>>();

        private static readonly object Lock = new object();

        /// <summary>
        /// 获取一个可用代理
        /// </summary>
        /// <param name="key">分类标签，与Tester一同使用，默认为null</param>
        /// <returns></returns>
        public static ProxyInfo GetIpProxy(string key)
        {
            List<ProxyInfo> ips = new List<ProxyInfo>();
            int times = 0;
            while (ips.Count < 0)
            {
                if (null != key)
                {
                    var tryGetAndReturn = Pool.TryGetAndReturn(key);
                    if (null != tryGetAndReturn)
                    {
                        ips.AddRange(tryGetAndReturn);
                    }
                }
                else
                {
                    lock (Lock)
                    {
                        ips.AddRange(IpList);
                    }
                }
                if (ips.Count < 0)
                {
                    ClassifyIpList();
                }
                times++;

                if (times > 20)
                {
                    Log.Error("获取免费IP代理失败");
                    break;
                }
            }

            if (ips.Count > 0)
            {
                return ips[0];
            }

            throw new Exception("获取免费IP代理失败");
        }

        public static void RemoveProxy(string key, ProxyInfo info)
        {
            var tryGetAndReturn = Pool.TryGetAndReturn(key);
            if (tryGetAndReturn.Contains(info))
            {
                tryGetAndReturn.Remove(info);
            }
        }

        /// <summary>
        /// 增加一个代理地址测试器，该测试器由定时任务调用，测试成功将按key分类
        /// </summary>
        /// <param name="key">分类标签</param>
        /// <param name="tester"></param>
        public static void AddTester(string key, Tester tester)
        {
            if (Testers.ContainsKey(key))
            {
                Testers.Remove(key);
            }

            Testers.Add(key, tester);
            var proxyInfos = new List<ProxyInfo>();

            lock (Lock)
            {
                proxyInfos.AddRange(IpList);
            }

            var list = proxyInfos.Select(m =>
            {
                var dateTime = DateTime.Now;
                var pass = tester.Invoke(m.Ip, m.Port);
                var delay = (DateTime.Now - dateTime).Ticks;
                var proxyInfo = new ProxyInfo(m.Ip, m.Port) {Delay = pass ? delay : 0};
                return proxyInfo;
            }).Where(m => m.Delay > 0).OrderBy(m => m.Delay).ToList();

            Log.Info($"增加测试器key={key}\t可用IP数量 count={list.Count}\tIP列表{list.toString()}");

            Pool.AddOrUpdate(key, list, (l, r) => list);
        }

        /// <summary>
        /// 获取
        /// </summary>
        public static void CrawlingProxyPool()
        {
//            CrawlIpHaiProxy();
            CrawlXiCiProxy();
//            CrawlJiSuProxy();
        }

        /// <summary>
        /// 分类并测试代理是否可用
        /// </summary>
        public static void ClassifyIpList()
        {
            while (IpList.Count == 0)
            {
                return;
            }

            //测试代理IP是否可以用
            FitnessForUseTest();

            //测试代理IP对目标网站是否可用
            var infos = new List<ProxyInfo>();
            lock (Lock)
            {
                infos.AddRange(IpList);
            }

            var poolKeys = Pool.Keys;
            foreach (var proxyInfo in infos)
            {
                foreach (var key in poolKeys)
                {
                    var pool = Pool.GetOrAdd(key, new List<ProxyInfo>());
                    var tester = Testers.TryGetAndReturn(key);

                    var pass = tester.Invoke(proxyInfo.Ip, proxyInfo.Port);

                    if (pass)
                    {
                        pool.Add(proxyInfo);
                    }
                }
            }
            try
            {
                IpPoolDao.BatchSaveWord(IpList.Select(m => m.ToString()).ToList());
            }
            catch (Exception e)
            {
                Log.Error(e);
            }
        }

        /// <summary>
        /// IP代理可用性测试
        /// </summary>
        private static void FitnessForUseTest()
        {
            var infos = new List<ProxyInfo>();
            lock (Lock)
            {
                infos.AddRange(IpList);
            }
            //测试代理IP是否可以用
            var proxyInfos = infos.Where(m => TestBaidu(m) > 0).ToList();

            lock (Lock)
            {
                //清理通用无用IP
                IpList.Clear();
                IpList.AddRange(proxyInfos);
            }

            //清理专用无用IP
            foreach (var proxyInfo in proxyInfos)
            {
                foreach (var poolList in Pool)
                {
                    if (poolList.Value.Contains(proxyInfo))
                    {
                        poolList.Value.Remove(proxyInfo);
                    }
                }
            }
        }

        /// <summary>
        /// IP可用性测试
        /// </summary>
        /// <param name="proxyInfo"></param>
        /// <returns></returns>
        private static long TestBaidu(ProxyInfo proxyInfo)
        {
            string url = "https://www.baidu.com/";
            var webProxy = new WebProxy(proxyInfo.Ip, proxyInfo.Port);
            HttpWebRequest request = (HttpWebRequest) WebRequest.Create(url);
            request.Method = WebRequestMethods.Http.Get;
            request.ContinueTimeout = 500;
            request.Timeout = 500;
            request.ReadWriteTimeout = 500;

            request.KeepAlive = false;
            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
            request.UserAgent =
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36";
            request.Proxy = webProxy;
            WebResponse webResponse = null;
            Stream myResponseStream = null;
            StreamReader myStreamReader = null;
            string html = "";
            var dateTime = DateTime.Now;
            try
            {
                var responseAsync = request.GetResponseAsync();
                responseAsync.Wait(500);
                if (responseAsync.Status == TaskStatus.RanToCompletion)
                {
                    webResponse = responseAsync.Result;
                    myResponseStream = webResponse.GetResponseStream();
                    myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8"));
                    html = myStreamReader.ReadToEnd();
                }
            }
            catch (Exception e)
            {
                Log.Error(e);
                return 0;
            }
            finally
            {
                myStreamReader?.Close();
                myResponseStream?.Close();
            }
            return html.contains("百度一下") ? (DateTime.Now - dateTime).Ticks : 0;
        }

        /// <summary>
        /// 更新IP列表
        /// </summary>
        public static void RefreshIpList()
        {
            string url = "http://api3.xiguadaili.com/ip/?tid=559629680038510&num=50&sortby=time&filter=on";
            HttpWebRequest request = (HttpWebRequest) WebRequest.Create(url);
            request.Method = WebRequestMethods.Http.Get;
            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
            request.UserAgent =
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36";
            var webResponse = request.GetResponse();

            Stream myResponseStream = null;
            StreamReader myStreamReader = null;
            string html = null;
            try
            {
                myResponseStream = webResponse.GetResponseStream();
                myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8"));
                html = myStreamReader.ReadToEnd();
            }
            catch (Exception e)
            {
                Log.Error(e);
                throw;
            }
            finally
            {
                myStreamReader?.Close();
                myResponseStream?.Close();
            }

            if (null != html)
            {
                var strings = html.Split('\n');
                var proxyInfos = strings.Select(m =>
                {
                    var split = m.Split(':');
                    return new ProxyInfo(split[0], int.Parse(split[1]));
                }).ToList();

                lock (Lock)
                {
                    IpList.AddRange(proxyInfos);
                }
            }
        }

        #region 代理池

        /// <summary>
        /// 爬取IP海免费代理
        /// </summary>
        /// <param name="tryTime"></param>
        public static void CrawlIpHaiProxy(int tryTime = 0)
        {
            //var proxy = GetProxy();
            var url = "http://www.iphai.com/free/ng";
            var html = "";
            try
            {
                html = HttpController.GetAsync(url).Result;
            }
            catch (Exception ex)
            {
                if (tryTime > 5)
                {
                    return;
                }
                CrawlIpHaiProxy(tryTime + 1);
                return;
            }

            var nodeRoot = HtmlController.GetHtmlDocument(html).DocumentNode;
            var listNodeTable = HtmlController.GetHtmlNodeList(nodeRoot, "table");
            if (listNodeTable.Count > 0)
            {
                var nodeTable = listNodeTable[0];
                var listNodeTr = HtmlController.GetHtmlNodeList(nodeTable, "tr");
                int addNum = 0;
                for (int i = 1; i < listNodeTr.Count; i++)
                {
                    var nodeTr = listNodeTr[i];
                    var listNodeTd = HtmlController.GetHtmlNodeList(nodeTr, "td");
                    if (listNodeTd[2].InnerText.Trim() == "高匿" && listNodeTd[3].InnerText.Trim() != "HTTPS")
                    {
                        var proxyInfo = new ProxyInfo(listNodeTd[0].InnerText.Trim(),
                            Convert.ToInt32(listNodeTd[1].InnerText.Trim()))
                        {
                            Location = listNodeTd[4].InnerText.Trim(),
                            FailTime = 0,
                            CanUse = true
                        };
                        var b = RedisController.AddMyProxy(proxyInfo);
                        if (b)
                        {
                            addNum++;
                        }
                    }
                }
                Log.Info($"新增代理 {addNum} 个。");
            }
            else
            {
                Log.Error($"{html}|尝试爬取次数：{tryTime}");
                if (tryTime > 5)
                {
                    return;
                }
                CrawlIpHaiProxy(tryTime + 1);
            }
        }

        /// <summary>
        /// 爬取西刺免费代理
        /// </summary>
        /// <param name="tryTime"></param>
        public static void CrawlXiCiProxy(int tryTime = 0)
        {
            //var proxy = GetProxy();
            var url = "http://www.xicidaili.com/";
            var html = "";
            try
            {
                html = HttpController.GetAsync(url).Result;
            }
            catch (Exception ex)
            {
                if (tryTime > 5)
                {
                    return;
                }
                CrawlXiCiProxy(tryTime + 1);
                return;
            }

            var nodeRoot = HtmlController.GetHtmlDocument(html).DocumentNode;
            var listNodeTable = HtmlController.GetHtmlNodeList(nodeRoot, "table");
            if (listNodeTable.Count > 0)
            {
                var nodeTable = listNodeTable[0];
                var listNodeTr = HtmlController.GetHtmlNodeList(nodeTable, "tr");
                int addNum = 0;
                for (int i = 1; i < listNodeTr.Count; i++)
                {
                    var nodeTr = listNodeTr[i];
                    var listNodeTd = HtmlController.GetHtmlNodeList(nodeTr, "td");
                    if (listNodeTd.Count == 8)
                    {
                        if ("高匿".Equals(listNodeTd[4].InnerText.Trim())  && "HTTP".Equals(listNodeTd[5].InnerText.Trim()))
                        {
                            var ProxyInfo = new ProxyInfo(listNodeTd[1].InnerText.Trim(),
                                Convert.ToInt32(listNodeTd[2].InnerText.Trim()))
                            {
                                Location = listNodeTd[3].InnerText.Trim(),
                                FailTime = 0,
                                CanUse = true
                            };
                            var b = RedisController.AddMyProxy(ProxyInfo);
                            if (b)
                            {
                                addNum++;
                            }
                        }
                    }
                }
                Log.Info($"新增代理 {addNum} 个。");
            }
            else
            {
                Log.Error($"{html}|尝试爬取次数：{tryTime}");
                if (tryTime > 5)
                {
                    return;
                }
                CrawlXiCiProxy(tryTime + 1);
            }
        }

        /// <summary>
        /// 爬取66代理免费代理
        /// </summary>
        /// <param name="tryTime"></param>
        public static void Crawl66Proxy(int tryTime = 0)
        {
            var proxy = GetProxy();
            var url =
                "http://www.66ip.cn/nmtq.php?getnum=10&isp=0&anonymoustype=4&start=&ports=&export=&ipaddress=&area=0&proxytype=0&api=66ip";
            var html = "";
            try
            {
                html = HttpController.GetAsyncByProxy(url, proxy.Ip, proxy.Port, Encoding.Default).Result;
            }
            catch (Exception ex)
            {
                UpdateFailProxy(proxy);
                Log.Error($"{ex.Message}|{proxy.Ip}|{proxy.Port}|尝试爬取次数：{tryTime}");
                if (tryTime > 5)
                {
                    return;
                }
                Crawl66Proxy(tryTime + 1);
                return;
            }
            html = html.Replace("\r", "").Replace("\n", "").Replace("\t", "");
            var list = Regex.Split(html, "<br />", RegexOptions.IgnoreCase).ToList();
            list.RemoveAt(0);
            list.RemoveAt(list.Count - 1);

            int addNum = 0;
            foreach (var s in list)
            {
                var a = s.Trim().Split(':');
                var ip = a[0];
                var port = Convert.ToInt32(a[1]);

                var ProxyInfo = new ProxyInfo(ip, port)
                {
                    FailTime = 0,
                    CanUse = true
                };

                var b = RedisController.AddMyProxy(ProxyInfo);
                if (b)
                {
                    addNum++;
                }
            }
            Log.Info($"新增代理 {addNum} 个。");
        }

        /// <summary>
        /// 爬取极速代理免费代理
        /// </summary>
        /// <param name="tryTime"></param>
        public static void CrawlJiSuProxy(int tryTime = 0)
        {
            var proxy = GetProxy();
            var url = "http://superfastip.com/welcome/getapi";
            var html = "";
            try
            {
                html = HttpController.GetAsyncByProxy(url, proxy.Ip, proxy.Port, Encoding.UTF8).Result;
                html = ToolController.UnicodeDecode(html);
            }
            catch (Exception ex)
            {
                UpdateFailProxy(proxy);
                Log.Error($"{ex.Message}|{proxy.Ip}|{proxy.Port}|尝试爬取次数：{tryTime}");
                if (tryTime > 5)
                {
                    return;
                }
                CrawlJiSuProxy(tryTime + 1);
                return;
            }

            var rsp = ToolController.JsonToObject<JiSuProxyRsp>(html);
            int addNum = 0;
            for (int i = 0; i < rsp.data.Count; i++)
            {
                var p = rsp.data[i];
                if (p[4] == "高匿" && p[5] == "HTTP")
                {
                    var ProxyInfo = new ProxyInfo(p[1].Trim(), Convert.ToInt32(p[2].Trim()))
                    {
                        Location = p[3].Trim(),
                        FailTime = 0,
                        CanUse = true
                    };
                    var b = RedisController.AddMyProxy(ProxyInfo);
                    if (b)
                    {
                        addNum++;
                    }
                }
            }
            Log.Info($"新增代理 {addNum} 个。");
        }

        /// <summary>
        /// 爬取米扑代理免费代理
        /// </summary>
        /// <param name="tryTime"></param>
        public static void CrawlMiPuProxy(int tryTime = 0)
        {
            var proxy = GetProxy();
            var url = "http://proxy.mimvp.com/free.php?proxy=in_hp";
            var html = "";
            try
            {
                html = HttpController.GetAsync(url).Result;
            }
            catch (Exception ex)
            {
                UpdateFailProxy(proxy);
                Log.Error($"{ex.Message}|{proxy.Ip}|{proxy.Port}|尝试爬取次数：{tryTime}");
                if (tryTime > 5)
                {
                    return;
                }
                CrawlMiPuProxy(tryTime + 1);
                return;
            }

            var nodeRoot = HtmlController.GetHtmlDocument(html).DocumentNode;
            var listNodeTable = HtmlController.GetHtmlNodeList(nodeRoot, "tbody");
            if (listNodeTable.Count > 0)
            {
                var nodeTable = listNodeTable[0];
                var listNodeTd = HtmlController.GetHtmlNodeList(nodeTable, "td");
                var objectNum = listNodeTd.Count / 10;

                int addNum = 0;
                for (int i = 0; i < objectNum; i++)
                {
                    var type = listNodeTd[10 * i + 3].InnerText.Trim();
                    var anonymous = listNodeTd[10 * i + 4].InnerText.Trim();
                    if (type != "HTTPS" && anonymous == "高匿")
                    {
                        var imgSrc = listNodeTd[10 * i + 2].ChildNodes[0].GetAttributeValue("src", "");
                        var imgName = imgSrc.Split('=')[2];
                        var ProxyInfo = new ProxyInfo(listNodeTd[10 * i + 1].InnerText.Trim(), ConvertMiPuPort(imgName))
                        {
                            Location = listNodeTd[10 * i + 5].InnerText.Trim(),
                            FailTime = 0,
                            CanUse = true
                        };
                        if (ProxyInfo.Port != -1)
                        {
                            var b = RedisController.AddMyProxy(ProxyInfo);
                            if (b)
                            {
                                addNum++;
                            }
                        }
                    }
                }
                Log.Info($"新增代理 {addNum} 个。");
            }
        }

        /// <summary>
        /// 转换米扑代理的图片端口为数字端口
        /// </summary>
        /// <param name="imgName"></param>
        /// <returns></returns>
        private static int ConvertMiPuPort(string imgName)
        {
            if (imgName.Contains("4vMpDgO0O"))
            {
                return 808;
            }
            else if (imgName.Contains("4vOpDg4"))
            {
                return 8888;
            }
            else if (imgName.Contains("1vMpzI4MQO0OO0O"))
            {
                return 53281;
            }
            else if (imgName.Contains("4vOpTk4"))
            {
                return 8998;
            }
            else if (imgName.Contains("4vMpAO0OO0O"))
            {
                return 80;
            }
            else if (imgName.Contains("4vMpTIz"))
            {
                return 8123;
            }
            else if (imgName.Contains("4vMpDgw"))
            {
                return 8080;
            }
            else if (imgName.Contains("4vMpTE4"))
            {
                return 8118;
            }
            else if (imgName.Contains("5vOpTk5"))
            {
                return 9999;
            }
            else if (imgName.Contains("zvMpTI4"))
            {
                return 3128;
            }
            else if (imgName.Contains("5vNpzk3"))
            {
                return 9797;
            }
            else if (imgName.Contains("5vMpDAw"))
            {
                return 9000;
            }
            else if (imgName.Contains("4vMpDg4"))
            {
                return 8088;
            }
            else if (imgName.Contains("4vOpDg5"))
            {
                return 8889;
            }
            else if (imgName.Contains("0vNpjYwMwO0OO0O"))
            {
                return 46603;
            }
            else
            {
                Log.Warn(imgName);
                RedisController.RedisTest.HashSet("MiPuPortImgName", imgName, false);
                return -1;
            }
        }

        /// <summary>
        /// 爬取讯代理免费代理
        /// </summary>
        /// <param name="tryTime"></param>
        public static void CrawlXunProxy(int tryTime = 0)
        {
            var proxy = GetProxy();
            var url = "http://www.xdaili.cn/ipagent//freeip/getFreeIps?page=1&rows=10";
            var html = "";
            try
            {
                //html = HttpController.GetAsyncByProxy(url, proxy.Ip, proxy.Port, Encoding.UTF8).Result;
                html = HttpController.GetAsync(url).Result;
                //RedisController.RedisTest.StringAddOrUpdate("xundaili", html);
                //html = RedisController.RedisTest.StringGet<string>("xundaili");
            }
            catch (Exception ex)
            {
                UpdateFailProxy(proxy);
                Log.Error($"{ex.Message}|{proxy.Ip}|{proxy.Port}|尝试爬取次数：{tryTime}");
                if (tryTime > 5)
                {
                    return;
                }
                CrawlXunProxy(tryTime + 1);
                return;
            }

            var rsp = ToolController.JsonToObject<XunProxyRsp>(html);

            int addNum = 0;
            foreach (var p in rsp.rows)
            {
                if (p.anony == "高匿" && p.type != "HTTPS")
                {
                    var proxyInfo = new ProxyInfo(p.ip, p.port)
                    {
                        Location = p.position,
                        FailTime = 0,
                        CanUse = true
                    };

                    var b = RedisController.AddMyProxy(proxyInfo);
                    if (b)
                    {
                        addNum++;
                    }
                }
            }
            Log.Info($"新增代理 {addNum} 个。");
        }

        /// <summary>
        /// 获取代理IP和端口
        /// </summary>
        /// <returns></returns>
        public static ProxyInfo GetProxy()
        {
            var listProxy = RedisController.GetAllMyProxy();
            var index = new Random().Next(0, listProxy.Count - 1);
            var proxy = listProxy[index];
            return proxy;
        }

        /// <summary>
        /// 更新代理失败次数
        /// </summary>
        /// <param name="proxyInfo"></param>
        public static void UpdateFailProxy(ProxyInfo proxyInfo)
        {
            proxyInfo.FailTime += 1;
            if (proxyInfo.FailTime >= 5)
            {
                proxyInfo.CanUse = false;
            }
            RedisController.UpdateMyProxy(proxyInfo);
        }

        #endregion

        public delegate bool Tester(string ip, int port);

        public static Dictionary<string, Tester> Testers = new Dictionary<string, Tester>();
    }
}